Speech Emotion Recognition with CNN¶
Data Exploration¶
In [1]:
# Import libraries
import librosa
import librosa.display
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from matplotlib.pyplot import specgram
import pandas as pd
import glob
from sklearn.metrics import confusion_matrix
import IPython.display as ipd # To play sound in the notebook
import os
import sys
import warnings
# ignore warnings
if not sys.warnoptions:
warnings.simplefilter("ignore")
warnings.filterwarnings("ignore", category=DeprecationWarning)
In [2]:
#for dirname, _, filenames in os.walk('/kaggle/input'):
# for filename in filenames:
# print(os.path.join(dirname, filename))
TESS = "/kaggle/input/toronto-emotional-speech-set-tess/tess toronto emotional speech set data/TESS Toronto emotional speech set data/"
RAV = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/"
SAVEE = "/kaggle/input/surrey-audiovisual-expressed-emotion-savee/ALL/"
CREMA = "/kaggle/input/cremad/AudioWAV/"
# Run one example
dir_list = os.listdir(SAVEE)
dir_list[0:5]
Out[2]:
Surrey Audio-Visual Expressed Emotion (SAVEE)¶
In [3]:
# Get the data location for SAVEE
dir_list = os.listdir(SAVEE)
# parse the filename to get the emotions
emotion=[]
path = []
for i in dir_list:
if i[-8:-6]=='_a':
emotion.append('male_angry')
elif i[-8:-6]=='_d':
emotion.append('male_disgust')
elif i[-8:-6]=='_f':
emotion.append('male_fear')
elif i[-8:-6]=='_h':
emotion.append('male_happy')
elif i[-8:-6]=='_n':
emotion.append('male_neutral')
elif i[-8:-6]=='sa':
emotion.append('male_sad')
elif i[-8:-6]=='su':
emotion.append('male_surprise')
else:
emotion.append('male_error')
path.append(SAVEE + i)
# Now check out the label count distribution
SAVEE_df = pd.DataFrame(emotion, columns = ['labels'])
SAVEE_df['source'] = 'SAVEE'
SAVEE_df = pd.concat([SAVEE_df, pd.DataFrame(path, columns = ['path'])], axis = 1)
SAVEE_df.labels.value_counts()
Out[3]:
In [4]:
# use the well known Librosa library for this task
fname = SAVEE + 'DC_f11.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[4]:
In [5]:
# Lets play a happy track
fname = SAVEE + 'DC_h11.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[5]:
Ryerson Audio-Visual Database of Emotional Speech and Song (RAVDESS)¶
In [6]:
dir_list = os.listdir(RAV)
dir_list.sort()
emotion = []
gender = []
path = []
for i in dir_list:
fname = os.listdir(RAV + i)
for f in fname:
part = f.split('.')[0].split('-')
emotion.append(int(part[2]))
temp = int(part[6])
if temp%2 == 0:
temp = "female"
else:
temp = "male"
gender.append(temp)
path.append(RAV + i + '/' + f)
RAV_df = pd.DataFrame(emotion)
RAV_df = RAV_df.replace({1:'neutral', 2:'neutral', 3:'happy', 4:'sad', 5:'angry', 6:'fear', 7:'disgust', 8:'surprise'})
RAV_df = pd.concat([pd.DataFrame(gender),RAV_df],axis=1)
RAV_df.columns = ['gender','emotion']
RAV_df['labels'] =RAV_df.gender + '_' + RAV_df.emotion
RAV_df['source'] = 'RAVDESS'
RAV_df = pd.concat([RAV_df,pd.DataFrame(path, columns = ['path'])],axis=1)
RAV_df = RAV_df.drop(['gender', 'emotion'], axis=1)
RAV_df.labels.value_counts()
Out[6]:
In [7]:
# Pick a fearful track
fname = RAV + 'Actor_14/03-01-06-02-02-02-14.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[7]:
In [8]:
# Pick a happy track
fname = RAV + 'Actor_14/03-01-03-02-02-02-14.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[8]:
Toronto Emotional Speech Set (TESS)¶
In [9]:
dir_list = os.listdir(TESS)
dir_list.sort()
dir_list
Out[9]:
In [10]:
path = []
emotion = []
for i in dir_list:
fname = os.listdir(TESS + i)
for f in fname:
if i == 'OAF_angry' or i == 'YAF_angry':
emotion.append('female_angry')
elif i == 'OAF_disgust' or i == 'YAF_disgust':
emotion.append('female_disgust')
elif i == 'OAF_Fear' or i == 'YAF_fear':
emotion.append('female_fear')
elif i == 'OAF_happy' or i == 'YAF_happy':
emotion.append('female_happy')
elif i == 'OAF_neutral' or i == 'YAF_neutral':
emotion.append('female_neutral')
elif i == 'OAF_Pleasant_surprise' or i == 'YAF_pleasant_surprised':
emotion.append('female_surprise')
elif i == 'OAF_Sad' or i == 'YAF_sad':
emotion.append('female_sad')
else:
emotion.append('Unknown')
path.append(TESS + i + "/" + f)
TESS_df = pd.DataFrame(emotion, columns = ['labels'])
TESS_df['source'] = 'TESS'
TESS_df = pd.concat([TESS_df,pd.DataFrame(path, columns = ['path'])],axis=1)
TESS_df.labels.value_counts()
Out[10]:
In [11]:
# lets play a fearful track
fname = TESS + 'YAF_fear/YAF_dog_fear.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[11]:
In [12]:
# lets play a happy track
fname = TESS + 'YAF_happy/YAF_dog_happy.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[12]:
Crowd Sourced Emotional Multimodal Actors Dataset (CREDMA-D)¶
In [13]:
dir_list = os.listdir(CREMA)
dir_list.sort()
print(dir_list[0:10])
In [14]:
gender = []
emotion = []
path = []
female = [1002,1003,1004,1006,1007,1008,1009,1010,1012,1013,1018,1020,1021,1024,1025,1028,1029,1030,1037,1043,1046,1047,1049,
1052,1053,1054,1055,1056,1058,1060,1061,1063,1072,1073,1074,1075,1076,1078,1079,1082,1084,1089,1091]
for i in dir_list:
part = i.split('_')
if int(part[0]) in female:
temp = 'female'
else:
temp = 'male'
gender.append(temp)
if part[2] == 'SAD' and temp == 'male':
emotion.append('male_sad')
elif part[2] == 'ANG' and temp == 'male':
emotion.append('male_angry')
elif part[2] == 'DIS' and temp == 'male':
emotion.append('male_disgust')
elif part[2] == 'FEA' and temp == 'male':
emotion.append('male_fear')
elif part[2] == 'HAP' and temp == 'male':
emotion.append('male_happy')
elif part[2] == 'NEU' and temp == 'male':
emotion.append('male_neutral')
elif part[2] == 'SAD' and temp == 'female':
emotion.append('female_sad')
elif part[2] == 'ANG' and temp == 'female':
emotion.append('female_angry')
elif part[2] == 'DIS' and temp == 'female':
emotion.append('female_disgust')
elif part[2] == 'FEA' and temp == 'female':
emotion.append('female_fear')
elif part[2] == 'HAP' and temp == 'female':
emotion.append('female_happy')
elif part[2] == 'NEU' and temp == 'female':
emotion.append('female_neutral')
else:
emotion.append('Unknown')
path.append(CREMA + i)
CREMA_df = pd.DataFrame(emotion, columns = ['labels'])
CREMA_df['source'] = 'CREMA'
CREMA_df = pd.concat([CREMA_df,pd.DataFrame(path, columns = ['path'])],axis=1)
CREMA_df.labels.value_counts()
Out[14]:
In [15]:
# use the well known Librosa library for this task
fname = CREMA + '1012_IEO_HAP_HI.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[15]:
In [16]:
# A fearful track
fname = CREMA + '1012_IEO_FEA_HI.wav'
data, sampling_rate = librosa.load(fname)
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
# Lets play the audio
ipd.Audio(fname)
Out[16]:
Combining the 4 Datasets¶
In [17]:
df = pd.concat([SAVEE_df, RAV_df, TESS_df, CREMA_df], axis = 0)
print(df.labels.value_counts())
df.head()
df.to_csv("Data_path.csv",index=False)
Feature Extraction¶
In [18]:
# Import our libraries
import librosa
import librosa.display
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.pyplot import specgram
import pandas as pd
import os
import IPython.display as ipd # To play sound in the notebook
In [19]:
# Source - RAVDESS; Gender - Female; Emotion - Angry
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_08/03-01-05-02-01-01-08.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
mfcc = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
# audio wave
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.waveplot(X, sr=sample_rate)
plt.title('Audio sampled at 44100 hrz')
# MFCC
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.specshow(mfcc, x_axis='time')
plt.ylabel('MFCC')
plt.colorbar()
ipd.Audio(path)
Out[19]:
In [20]:
# Source - RAVDESS; Gender - Male; Emotion - Angry
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_09/03-01-05-01-01-01-09.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
mfcc = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
# audio wave
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.waveplot(X, sr=sample_rate)
plt.title('Audio sampled at 44100 hrz')
# MFCC
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.specshow(mfcc, x_axis='time')
plt.ylabel('MFCC')
plt.colorbar()
ipd.Audio(path)
Out[20]:
In [21]:
# Source - RAVDESS; Gender - Female; Emotion - Happy
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_12/03-01-03-01-02-01-12.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
mfcc = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
# audio wave
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.waveplot(X, sr=sample_rate)
plt.title('Audio sampled at 44100 hrz')
# MFCC
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.specshow(mfcc, x_axis='time')
plt.ylabel('MFCC')
plt.colorbar()
ipd.Audio(path)
Out[21]:
In [22]:
# Source - RAVDESS; Gender - Male; Emotion - Happy
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_11/03-01-03-01-02-02-11.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
mfcc = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
# audio wave
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.waveplot(X, sr=sample_rate)
plt.title('Audio sampled at 44100 hrz')
# MFCC
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
librosa.display.specshow(mfcc, x_axis='time')
plt.ylabel('MFCC')
plt.colorbar()
ipd.Audio(path)
Out[22]:
In [23]:
# Source - RAVDESS; Gender - Female; Emotion - Angry
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_08/03-01-05-02-01-01-08.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
female = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
female = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13), axis=0)
print(len(female))
# Source - RAVDESS; Gender - Male; Emotion - Angry
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_09/03-01-05-01-01-01-09.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
male = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
male = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13), axis=0)
print(len(male))
# audio wave
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
plt.plot(female, label='female')
plt.plot(male, label='male')
plt.legend()
Out[23]:
In [24]:
# Source - RAVDESS; Gender - Female; Emotion - happy
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_12/03-01-03-01-02-01-12.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
female = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
female = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13), axis=0)
print(len(female))
# Source - RAVDESS; Gender - Male; Emotion - happy
path = "/kaggle/input/ravdess-emotional-speech-audio/audio_speech_actors_01-24/Actor_11/03-01-03-01-02-02-11.wav"
X, sample_rate = librosa.load(path, res_type='kaiser_fast',duration=2.5,sr=22050*2,offset=0.5)
male = librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13)
male = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13), axis=0)
print(len(male))
# Plot the two audio waves together
plt.figure(figsize=(20, 15))
plt.subplot(3,1,1)
plt.plot(female, label='female')
plt.plot(male, label='male')
plt.legend()
Out[24]:
Data Preparation & Processing¶
In [25]:
# Importing required libraries
# Keras
import keras
from keras import regularizers
from keras.preprocessing import sequence
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential, Model, model_from_json
from keras.layers import Dense, Embedding, LSTM
from keras.layers import Input, Flatten, Dropout, Activation, BatchNormalization
from keras.layers import Conv1D, MaxPooling1D, AveragePooling1D
from keras.utils import np_utils, to_categorical
from keras.callbacks import ModelCheckpoint
# sklearn
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
# Other
import librosa
import librosa.display
import json
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from matplotlib.pyplot import specgram
import pandas as pd
import seaborn as sns
import glob
import os
import pickle
import IPython.display as ipd # To play sound in the notebook
In [26]:
# lets pick up the meta-data that we got from our first part of the Kernel
ref = pd.read_csv("/kaggle/input/datapath/Data_path.csv")
ref.head()
Out[26]:
In [27]:
# Note this takes a couple of minutes (~10 mins) as we're iterating over 4 datasets
df = pd.DataFrame(columns=['feature'])
# loop feature extraction over the entire dataset
counter=0
for index,path in enumerate(ref.path):
X, sample_rate = librosa.load(path
, res_type='kaiser_fast'
,duration=2.5
,sr=44100
,offset=0.5
)
sample_rate = np.array(sample_rate)
# mean as the feature. Could do min and max etc as well.
mfccs = np.mean(librosa.feature.mfcc(y=X,
sr=sample_rate,
n_mfcc=13),
axis=0)
df.loc[counter] = [mfccs]
counter=counter+1
# Check a few records to make sure its processed successfully
print(len(df))
df.head()
Out[27]:
In [28]:
# Now extract the mean bands to its own feature columns
df = pd.concat([ref,pd.DataFrame(df['feature'].values.tolist())],axis=1)
df[:5]
Out[28]:
In [29]:
# replace NA with 0
df=df.fillna(0)
print(df.shape)
df[:5]
Out[29]:
Splitting the Dataset into Training and Test¶
In [30]:
# Split between train and test
X_train, X_test, y_train, y_test = train_test_split(df.drop(['path','labels','source'],axis=1)
, df.labels
, test_size=0.25
, shuffle=True
, random_state=42
)
# Lets see how the data present itself before normalisation
X_train[150:160]
Out[30]:
In [31]:
# Lts do data normalization
mean = np.mean(X_train, axis=0)
std = np.std(X_train, axis=0)
X_train = (X_train - mean)/std
X_test = (X_test - mean)/std
# Check the dataset now
X_train[150:160]
Out[31]:
In [32]:
# Lets few preparation steps to get it into the correct format for Keras
X_train = np.array(X_train)
y_train = np.array(y_train)
X_test = np.array(X_test)
y_test = np.array(y_test)
# one hot encode the target
lb = LabelEncoder()
y_train = np_utils.to_categorical(lb.fit_transform(y_train))
y_test = np_utils.to_categorical(lb.fit_transform(y_test))
print(X_train.shape)
print(lb.classes_)
#print(y_train[0:10])
#print(y_test[0:10])
# Pickel the lb object for future use
filename = 'labels'
outfile = open(filename,'wb')
pickle.dump(lb,outfile)
outfile.close()
In [33]:
X_train = np.expand_dims(X_train, axis=2)
X_test = np.expand_dims(X_test, axis=2)
X_train.shape
Out[33]:
Modelling¶
In [34]:
# New model
model = Sequential()
model.add(Conv1D(256, 8, padding='same',input_shape=(X_train.shape[1],1))) # X_train.shape[1] = No. of Columns
model.add(Activation('relu'))
model.add(Conv1D(256, 8, padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.25))
model.add(MaxPooling1D(pool_size=(8)))
model.add(Conv1D(128, 8, padding='same'))
model.add(Activation('relu'))
model.add(Conv1D(128, 8, padding='same'))
model.add(Activation('relu'))
model.add(Conv1D(128, 8, padding='same'))
model.add(Activation('relu'))
model.add(Conv1D(128, 8, padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dropout(0.25))
model.add(MaxPooling1D(pool_size=(8)))
model.add(Conv1D(64, 8, padding='same'))
model.add(Activation('relu'))
model.add(Conv1D(64, 8, padding='same'))
model.add(Activation('relu'))
model.add(Flatten())
model.add(Dense(14)) # Target class number
model.add(Activation('softmax'))
# opt = keras.optimizers.SGD(lr=0.0001, momentum=0.0, decay=0.0, nesterov=False)
# opt = keras.optimizers.Adam(lr=0.0001)
opt = keras.optimizers.rmsprop(lr=0.00001, decay=1e-6)
model.summary()
Description of CNN-Model¶
In [35]:
model.compile(loss='categorical_crossentropy', optimizer=opt,metrics=['accuracy'])
model_history=model.fit(X_train, y_train, batch_size=16, epochs=100, validation_data=(X_test, y_test))
In [36]:
plt.plot(model_history.history['loss'])
plt.plot(model_history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
In [37]:
# Save model and weights
model_name = 'Emotion_Model.h5'
save_dir = os.path.join(os.getcwd(), 'saved_models')
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
model_path = os.path.join(save_dir, model_name)
model.save(model_path)
print('Save model and weights at %s ' % model_path)
# Save the model to disk
model_json = model.to_json()
with open("model_json.json", "w") as json_file:
json_file.write(model_json)
In [38]:
# loading json and model architecture
json_file = open('model_json.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("saved_models/Emotion_Model.h5")
print("Loaded model from disk")
# Keras optimiser
opt = keras.optimizers.rmsprop(lr=0.00001, decay=1e-6)
loaded_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
score = loaded_model.evaluate(X_test, y_test, verbose=0)
print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1]*100))
In [39]:
preds = loaded_model.predict(X_test,
batch_size=16,
verbose=1)
preds=preds.argmax(axis=1)
preds
Out[39]:
In [40]:
# predictions
preds = preds.astype(int).flatten()
preds = (lb.inverse_transform((preds)))
preds = pd.DataFrame({'predictedvalues': preds})
# Actual labels
actual=y_test.argmax(axis=1)
actual = actual.astype(int).flatten()
actual = (lb.inverse_transform((actual)))
actual = pd.DataFrame({'actualvalues': actual})
# Lets combined both of them into a single dataframe
finaldf = actual.join(preds)
finaldf[170:180]
Out[40]:
In [41]:
# Write out the predictions to disk
finaldf.to_csv('Predictions.csv', index=False)
finaldf.groupby('predictedvalues').count()
Out[41]:
Emotion vs Gender Accuracy¶
In [42]:
# the confusion matrix heat map plot
def print_confusion_matrix(confusion_matrix, class_names, figsize = (10,7), fontsize=14):
"""Prints a confusion matrix, as returned by sklearn.metrics.confusion_matrix, as a heatmap.
Arguments
---------
confusion_matrix: numpy.ndarray
The numpy.ndarray object returned from a call to sklearn.metrics.confusion_matrix.
Similarly constructed ndarrays can also be used.
class_names: list
An ordered list of class names, in the order they index the given confusion matrix.
figsize: tuple
A 2-long tuple, the first value determining the horizontal size of the ouputted figure,
the second determining the vertical size. Defaults to (10,7).
fontsize: int
Font size for axes labels. Defaults to 14.
Returns
-------
matplotlib.figure.Figure
The resulting confusion matrix figure
"""
df_cm = pd.DataFrame(
confusion_matrix, index=class_names, columns=class_names,
)
fig = plt.figure(figsize=figsize)
try:
heatmap = sns.heatmap(df_cm, annot=True, fmt="d")
except ValueError:
raise ValueError("Confusion matrix values must be integers.")
heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=fontsize)
heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=fontsize)
plt.ylabel('True label')
plt.xlabel('Predicted label')
# Gender recode function
def gender(row):
if row == 'female_disgust' or 'female_fear' or 'female_happy' or 'female_sad' or 'female_surprise' or 'female_neutral':
return 'female'
elif row == 'male_angry' or 'male_fear' or 'male_happy' or 'male_sad' or 'male_surprise' or 'male_neutral' or 'male_disgust':
return 'male'
In [43]:
# Get the predictions file
finaldf = pd.read_csv("Predictions.csv")
classes = finaldf.actualvalues.unique()
classes.sort()
# Confusion matrix
c = confusion_matrix(finaldf.actualvalues, finaldf.predictedvalues)
print(accuracy_score(finaldf.actualvalues, finaldf.predictedvalues))
print_confusion_matrix(c, class_names = classes)
In [44]:
# Classification report
classes = finaldf.actualvalues.unique()
classes.sort()
print(classification_report(finaldf.actualvalues, finaldf.predictedvalues, target_names=classes))
Gender Accuracy¶
In [45]:
modidf = finaldf
modidf['actualvalues'] = finaldf.actualvalues.replace({'female_angry':'female'
, 'female_disgust':'female'
, 'female_fear':'female'
, 'female_happy':'female'
, 'female_sad':'female'
, 'female_surprise':'female'
, 'female_neutral':'female'
, 'male_angry':'male'
, 'male_fear':'male'
, 'male_happy':'male'
, 'male_sad':'male'
, 'male_surprise':'male'
, 'male_neutral':'male'
, 'male_disgust':'male'
})
modidf['predictedvalues'] = finaldf.predictedvalues.replace({'female_angry':'female'
, 'female_disgust':'female'
, 'female_fear':'female'
, 'female_happy':'female'
, 'female_sad':'female'
, 'female_surprise':'female'
, 'female_neutral':'female'
, 'male_angry':'male'
, 'male_fear':'male'
, 'male_happy':'male'
, 'male_sad':'male'
, 'male_surprise':'male'
, 'male_neutral':'male'
, 'male_disgust':'male'
})
classes = modidf.actualvalues.unique()
classes.sort()
# Confusion matrix
c = confusion_matrix(modidf.actualvalues, modidf.predictedvalues)
print(accuracy_score(modidf.actualvalues, modidf.predictedvalues))
print_confusion_matrix(c, class_names = classes)
In [46]:
# Classification report
classes = modidf.actualvalues.unique()
classes.sort()
print(classification_report(modidf.actualvalues, modidf.predictedvalues, target_names=classes))
Emotion Accuracy¶
In [47]:
modidf = pd.read_csv("Predictions.csv")
modidf['actualvalues'] = modidf.actualvalues.replace({'female_angry':'angry'
, 'female_disgust':'disgust'
, 'female_fear':'fear'
, 'female_happy':'happy'
, 'female_sad':'sad'
, 'female_surprise':'surprise'
, 'female_neutral':'neutral'
, 'male_angry':'angry'
, 'male_fear':'fear'
, 'male_happy':'happy'
, 'male_sad':'sad'
, 'male_surprise':'surprise'
, 'male_neutral':'neutral'
, 'male_disgust':'disgust'
})
modidf['predictedvalues'] = modidf.predictedvalues.replace({'female_angry':'angry'
, 'female_disgust':'disgust'
, 'female_fear':'fear'
, 'female_happy':'happy'
, 'female_sad':'sad'
, 'female_surprise':'surprise'
, 'female_neutral':'neutral'
, 'male_angry':'angry'
, 'male_fear':'fear'
, 'male_happy':'happy'
, 'male_sad':'sad'
, 'male_surprise':'surprise'
, 'male_neutral':'neutral'
, 'male_disgust':'disgust'
})
classes = modidf.actualvalues.unique()
classes.sort()
# Confusion matrix
c = confusion_matrix(modidf.actualvalues, modidf.predictedvalues)
print(accuracy_score(modidf.actualvalues, modidf.predictedvalues))
print_confusion_matrix(c, class_names = classes)
In [48]:
# Classification report
classes = modidf.actualvalues.unique()
classes.sort()
print(classification_report(modidf.actualvalues, modidf.predictedvalues, target_names=classes))
Testing the Model on External Audio¶
In [49]:
from keras.models import Sequential, Model, model_from_json
import matplotlib.pyplot as plt
import keras
import pickle
import wave # !pip install wave
import os
import pandas as pd
import numpy as np
import sys
import warnings
import librosa
import librosa.display
import IPython.display as ipd # To play sound in the notebook
# ignore warnings
if not sys.warnoptions:
warnings.simplefilter("ignore")
In [50]:
data, sampling_rate = librosa.load('/kaggle/input/externaltest/DC_d02.wav')
ipd.Audio('/kaggle/input/externaltest/DC_d02.wav')
Out[50]:
In [51]:
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
Out[51]:
In [52]:
# loading json and model architecture
json_file = open('model_json.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("saved_models/Emotion_Model.h5")
print("Loaded model from disk")
# Keras optimiser
opt = keras.optimizers.rmsprop(lr=0.00001, decay=1e-6)
loaded_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
In [53]:
# Lets transform the dataset so we can apply the predictions
X, sample_rate = librosa.load('/kaggle/input/externaltest/DC_d02.wav'
,res_type='kaiser_fast'
,duration=2.5
,sr=44100
,offset=0.5
)
sample_rate = np.array(sample_rate)
mfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13),axis=0)
newdf = pd.DataFrame(data=mfccs).T
newdf
Out[53]:
Predictions¶
In [54]:
# Apply predictions
newdf= np.expand_dims(newdf, axis=2)
newpred = loaded_model.predict(newdf,
batch_size=16,
verbose=1)
newpred
Out[54]:
In [55]:
filename = '/kaggle/input/labels/labels'
infile = open(filename,'rb')
lb = pickle.load(infile)
infile.close()
# Get the final predicted label
final = newpred.argmax(axis=1)
final = final.astype(int).flatten()
final = (lb.inverse_transform((final)))
print(final) #emo(final) #gender(final)
Second Test¶
In [56]:
data, sampling_rate = librosa.load('/kaggle/input/externaltest/DC_d06.wav')
ipd.Audio('/kaggle/input/externaltest/DC_d06.wav')
Out[56]:
In [57]:
plt.figure(figsize=(15, 5))
librosa.display.waveplot(data, sr=sampling_rate)
Out[57]:
In [58]:
# loading json and model architecture
json_file = open('model_json.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("saved_models/Emotion_Model.h5")
print("Loaded model from disk")
# Keras optimiser
opt = keras.optimizers.rmsprop(lr=0.00001, decay=1e-6)
loaded_model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
In [59]:
# Lets transform the dataset so we can apply the predictions
X, sample_rate = librosa.load('/kaggle/input/externaltest/DC_d06.wav'
,res_type='kaiser_fast'
,duration=2.5
,sr=44100
,offset=0.5
)
sample_rate = np.array(sample_rate)
mfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=13),axis=0)
newdf = pd.DataFrame(data=mfccs).T
newdf
Out[59]:
In [60]:
# Apply predictions
newdf= np.expand_dims(newdf, axis=2)
newpred = loaded_model.predict(newdf,
batch_size=16,
verbose=1)
newpred
Out[60]:
In [61]:
filename = '/kaggle/input/labels/labels'
infile = open(filename,'rb')
lb = pickle.load(infile)
infile.close()
# Get the final predicted label
final = newpred.argmax(axis=1)
final = final.astype(int).flatten()
final = (lb.inverse_transform((final)))
print(final) #emo(final) #gender(final)